{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup, SoupStrainer, BeautifulStoneSoup\n",
    "import datetime\n",
    "import unicodedata\n",
    "import requests\n",
    "import pandas as pd\n",
    "import quandl\n",
    "import config\n",
    "import dateutil.relativedelta\n",
    "import re\n",
    "import numpy as np\n",
    "import pandas_market_calendars as mcal"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table of Contents\n",
    "1. Define Functions <br>\n",
    "    a. Get list of SEC docs for CIK<br>\n",
    "    b. Extract text, date from each link<br>\n",
    "    c. Get price given ticker, date from Quandl\n",
    "    d. Get movement given ticker, date\n",
    "    e. Get index movement\n",
    "    f. Check if date is a weekday, and if necessary, adjust to Friday before\n",
    "    g. Calculate dates for month before, quarter before, year before for eent mvoement calculations\n",
    "2. Download 8Ks\n",
    "3. Download Stock Movements\n",
    "### Process\n",
    "1. For each ticker in S&P 500:<br>\n",
    "    a. Get list of links to all 8Ks for a CIK<br>\n",
    "    b. Extract and clean up corpus<br>\n",
    "    c. Extract date and time of release<br>\n",
    "2. Stock Movements<br>\n",
    "    a. Calculate 1 Day movement(before and after release)<br>\n",
    "    b. Calculate 1 Month, 1 Quarter, 1 Year moving averages<br>\n",
    "    c. Get VIX at publication release<br>\n",
    "3. NLP\n",
    "\n",
    "### Get Corpus Info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Returns Dataframe of document links for a given CIK\n",
    "def get_sec_docs(cik,ticker):\n",
    "    base_url = \"https://www.sec.gov/cgi-bin/browse-edgar\"\n",
    "    inputted_cik = cik\n",
    "    payload = {\n",
    "        \"action\" : \"getcompany\",\n",
    "        \"CIK\" : inputted_cik,\n",
    "        \"type\" : \"8-K\",\n",
    "        \"output\":\"xml\",\n",
    "        \"dateb\" : \"20180401\",\n",
    "        #\"count\" : \"100\",\n",
    "        #\"owner\" : \"include\"\n",
    "    }\n",
    "    sec_response = requests.get(url=base_url,params=payload)\n",
    "    soup = BeautifulSoup(sec_response.text,'lxml')\n",
    "    url_list = soup.findAll('filinghref')\n",
    "    html_list = []\n",
    "    # Get html version of links\n",
    "    for link in url_list:\n",
    "        link = link.string\n",
    "        if link.split(\".\")[len(link.split(\".\"))-1] == 'htm':\n",
    "            txtlink = link + \"l\"\n",
    "            html_list.append(txtlink)\n",
    "\n",
    "    doc_list = []\n",
    "    doc_name_list = []\n",
    "    # Get links for txt versions of files\n",
    "    for k in range(len(html_list)):\n",
    "        txt_doc = html_list[k].replace(\"-index.html\",\".txt\")\n",
    "        doc_name = txt_doc.split(\"/\")[-1]\n",
    "        doc_list.append(txt_doc)\n",
    "        doc_name_list.append(doc_name)\n",
    "        # Create dataframe of CIK, doc name, and txt link\n",
    "    df = pd.DataFrame(\n",
    "        {\n",
    "        \"cik\" : [cik]*len(html_list),\n",
    "        \"ticker\" : [ticker]*len(html_list),\n",
    "        \"txt_link\" : doc_list,\n",
    "        \"doc_name\": doc_name_list\n",
    "        }\n",
    "    )\n",
    "    return df\n",
    "\n",
    "# Extracts text and submission datetime from document link\n",
    "def extract_text(link):\n",
    "    r = requests.get(link,timeout=30)\n",
    "    #Parse 8-K document\n",
    "    filing = BeautifulSoup(r.content,\"lxml\",from_encoding=\"ascii\")\n",
    "    #Extract datetime\n",
    "    submission_dt = filing.find(\"acceptance-datetime\").string[:14]\n",
    "    #Extract HTML sections\n",
    "    submission_dt = datetime.datetime.strptime(submission_dt,\"%Y%m%d%H%M%S\")\n",
    "    for section in filing.findAll(\"html\"):\n",
    "        #Remove tables\n",
    "        for table in section(\"table\"):\n",
    "            table.decompose()\n",
    "        #Convert to unicode\n",
    "        section = unicodedata.normalize(\"NFKD\",section.text)\n",
    "        section = section.replace(\"\\t\",\" \").replace(\"\\n\",\" \").replace(\"/s\",\" \").replace(\"\\'\",\"'\")\n",
    "    filing = \"\".join((section))\n",
    "    \n",
    "    return filing, submission_dt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get Stock & Index Movements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# S&P 500 index data downloaded from Yahoo Finance GSPC\n",
    "gspc_df = pd.read_csv(\"Data/gspc.csv\",parse_dates=['Date'],index_col=\"Date\")\n",
    "# Get VIX index data downloaded from Yahoo Finance\n",
    "vix_df = pd.read_csv(\"Data/vix.csv\",parse_dates=['Date'],index_col=\"Date\")\n",
    "#Authenticate with API KEY\n",
    "quandl.ApiConfig.api_key = config.api_key # YOUR API KEY HERE\n",
    "\n",
    "nyse = mcal.get_calendar('NYSE')\n",
    "nyse_holidays = nyse.holidays().holidays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Takes datetime object and ticker string, returns price (opening or closing)\n",
    "def get_historical_movements(ticker,period,release_date):\n",
    "   #1 Week\n",
    "    if period == \"week\":\n",
    "        e_start = release_date + datetime.timedelta(weeks=-1)\n",
    "        b_start = e_start\n",
    "        \n",
    "        e_end = release_date + dateutil.relativedelta.relativedelta(days=-1)\n",
    "        b_end = e_end\n",
    "        \n",
    "     #1 Month    \n",
    "    elif period == \"month\":\n",
    "        e_start = release_date + dateutil.relativedelta.relativedelta(months=-1)\n",
    "        b_start = e_start + dateutil.relativedelta.relativedelta(days=-5)\n",
    "        \n",
    "        e_end = release_date + dateutil.relativedelta.relativedelta(days=-1)\n",
    "        b_end = release_date + dateutil.relativedelta.relativedelta(days=-6)\n",
    "        \n",
    "    #1 Quarter\n",
    "    elif period == \"quarter\":\n",
    "        e_start = release_date + dateutil.relativedelta.relativedelta(months=-3)\n",
    "        b_start = e_start + dateutil.relativedelta.relativedelta(days=-10)\n",
    "        \n",
    "        e_end = release_date + dateutil.relativedelta.relativedelta(days=-1)\n",
    "        b_end = release_date + dateutil.relativedelta.relativedelta(days=-11)\n",
    "        \n",
    "    #1 Year\n",
    "    elif period == \"year\":\n",
    "        e_start = release_date + dateutil.relativedelta.relativedelta(years=-1)\n",
    "        b_start = e_start + dateutil.relativedelta.relativedelta(days=-20)\n",
    "        \n",
    "        e_end = release_date + dateutil.relativedelta.relativedelta(days=-1)\n",
    "        b_end = release_date + dateutil.relativedelta.relativedelta(days=-21)\n",
    "    else:\n",
    "        raise KeyError\n",
    "        \n",
    "    e_start = weekday_check(e_start)\n",
    "    b_start = weekday_check(b_start)\n",
    "    e_end = weekday_check(e_end)\n",
    "    b_end = weekday_check(b_end)\n",
    "    \n",
    "    start_price = get_quandl_data(ticker=ticker,start_date = b_start, end_date = e_start)\n",
    "    end_price = get_quandl_data(ticker=ticker,start_date = b_end, end_date = e_end)\n",
    "    stock_change = calculate_pct_change(end_price,start_price)\n",
    "    \n",
    "    start_index = get_index_price(start_date = b_start, end_date = e_start)\n",
    "    end_index = get_index_price(start_date = e_start, end_date = e_end)\n",
    "    index_change =  calculate_pct_change(end_index,start_index)\n",
    "    \n",
    "    normalized = stock_change - index_change\n",
    "    return normalized\n",
    "\n",
    "def get_quandl_data(ticker,start_date,end_date,market_open=False):\n",
    "    if market_open == True:\n",
    "        quandl_param = \"WIKI/\" + ticker + \".8\"  \n",
    "    else:\n",
    "        quandl_param = \"WIKI/\" + ticker + \".11\" \n",
    "   \n",
    "    end_date = datetime.datetime.strftime(end_date,\"%Y-%m-%d\") \n",
    "    start_date = datetime.datetime.strftime(start_date,\"%Y-%m-%d\")\n",
    "    price = quandl.get(quandl_param,start_date=start_date,end_date=end_date)\n",
    "    return price.mean()\n",
    "\n",
    "# Takes ticker, 8K release date, checks time of release and then calculate before and after price change\n",
    "def get_change(ticker,release_date):\n",
    "    market_close = release_date.replace(hour=16,minute=0,second=0)\n",
    "    market_open = release_date.replace(hour=9,minute=30,second=0)\n",
    "    \n",
    "# If report is released after market hours, take change of start date close and release date open\n",
    "    if release_date > market_close:\n",
    "        start_date = release_date\n",
    "        end_date = release_date + datetime.timedelta(days=1)\n",
    "        end_date = weekday_check(end_date)\n",
    "\n",
    "        price_before_release = get_quandl_data(ticker,start_date,market_open=False)\n",
    "        price_after_release = get_quandl_data(ticker,end_date,market_open=True)\n",
    "\n",
    "        index_before_release = get_index_price(start_date,start_date,market_open=False)\n",
    "        index_after_release = get_index_price(end_date,end_date,market_open=True)\n",
    "        \n",
    "        vix = vix_df.loc[vix_df.index == np.datetime64(start_date),\"Adj Close\"]\n",
    "\n",
    "    # If report is released before market hours, take change of start date's close and release date's open\n",
    "    elif release_date < market_open:\n",
    "        start_date = release_date + datetime.timedelta(days=-1)\n",
    "        start_date = weekday_check(start_date)\n",
    "        end_date = release_date\n",
    "\n",
    "        price_before_release = get_quandl_data(ticker,start_date,market_open=False)\n",
    "        price_after_release = get_quandl_data(ticker,end_date,market_open=True) \n",
    "\n",
    "        index_before_release = get_index_price(start_date,start_date,market_open=False)\n",
    "        index_after_release = get_index_price(end_date,end_date,market_open=True)\n",
    "        \n",
    "        vix = vix_df.loc[vix_df.index == np.datetime64(start_date),\"Adj Close\"]\n",
    "\n",
    "    # If report is released during market hours, use market close\n",
    "    else:\n",
    "        start_date = release_date\n",
    "        end_date = release_date\n",
    "        price_before_release = get_quandl_data(ticker,start_date,market_open=True)\n",
    "        price_after_release = get_quandl_data(ticker,end_date,market_open=False)\n",
    "\n",
    "        index_before_release = get_index_price(start_date,start_datemarket_open=True)\n",
    "        index_after_release = get_index_price(end_date,end_date,market_open=False)\n",
    "        \n",
    "        vix = vix_df.loc[vix_df.index == np.datetime64(start_date),\"Open\"]\n",
    "       \n",
    "    price_pct_change = calculate_pct_change(price_after_release,price_before_release)\n",
    "    index_pct_change = calculate_pct_change(index_after_release,index_before_release)\n",
    "    normalized_change = price_pct_change - index_pct_change\n",
    "    \n",
    "    \n",
    "    return normalized_change\n",
    "\n",
    "def get_index_price(start_date,end_date,market_open=False):\n",
    "    if market_open == True:\n",
    "        price = gspc_df.loc[(gspc_df.index >= np.datetime64(start_date)) & \n",
    "                         (gspc_df.index <= np.datetime64(end_date)),\"Open\"].mean()\n",
    "    else:\n",
    "        price = gspc_df.loc[(gspc_df.index >= np.datetime64(start_date)) & \n",
    "                         (gspc_df.index <= np.datetime64(end_date)),\"Adj Close\"].mean()\n",
    "    \n",
    "    return price\n",
    "\n",
    "def calculate_pct_change(end_value,start_value):\n",
    "    pct_change = (end_value - start_value) / start_value\n",
    "    pct_change = round(pct_change,4) * 100\n",
    "    return pct_change\n",
    "\n",
    "def weekday_check(date):\n",
    "    date = holiday_check(date)\n",
    "    \n",
    "    # If date is Saturday or Sunday, reset date to the preceding Friday\n",
    "    if date.isoweekday() == 6:\n",
    "        date = date + datetime.timedelta(days=-1)\n",
    "    elif date.isoweekday() == 7:\n",
    "        date = date + datetime.timedelta(days=-2)\n",
    "        \n",
    "    if date in nyse_holidays:\n",
    "        date = date + datetime.timedelta(days=-1) \n",
    "        \n",
    "    date = holiday_check(date)\n",
    "    return date\n",
    "\n",
    "def holiday_check(date):\n",
    "    if date in nyse_holidays:\n",
    "        date = date + datetime.timedelta(days=-1)\n",
    "    return date\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Features\n",
    "1. Pubication Date\n",
    "2. Publication Category\n",
    "3. Recent Movements (Normalized with S&P 500 Movements)<br>\n",
    "    a. 1 Day<br>\n",
    "    b. 1 Month (5 Day MA)<br>\n",
    "    c. 1 Quarter (10 Day MA)<br>\n",
    "    d. 1 Year(20 Day MA)<br>\n",
    "4. VIX at publication release\n",
    "5. Publicaion Corpus<br>\n",
    "    a. Unigram<br>\n",
    "    b. NMF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Security</th>\n",
       "      <th>SEC filings</th>\n",
       "      <th>GICS Sector</th>\n",
       "      <th>GICS Sub Industry</th>\n",
       "      <th>Address of Headquarters</th>\n",
       "      <th>Date first added[3][4]</th>\n",
       "      <th>CIK</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ticker symbol</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MMM</th>\n",
       "      <td>3M Company</td>\n",
       "      <td>reports</td>\n",
       "      <td>Industrials</td>\n",
       "      <td>Industrial Conglomerates</td>\n",
       "      <td>St. Paul, Minnesota</td>\n",
       "      <td>NaN</td>\n",
       "      <td>66740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABT</th>\n",
       "      <td>Abbott Laboratories</td>\n",
       "      <td>reports</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Health Care Equipment</td>\n",
       "      <td>North Chicago, Illinois</td>\n",
       "      <td>1964-03-31</td>\n",
       "      <td>1800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABBV</th>\n",
       "      <td>AbbVie Inc.</td>\n",
       "      <td>reports</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Pharmaceuticals</td>\n",
       "      <td>North Chicago, Illinois</td>\n",
       "      <td>2012-12-31</td>\n",
       "      <td>1551152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ACN</th>\n",
       "      <td>Accenture plc</td>\n",
       "      <td>reports</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>IT Consulting &amp; Other Services</td>\n",
       "      <td>Dublin, Ireland</td>\n",
       "      <td>2011-07-06</td>\n",
       "      <td>1467373</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ATVI</th>\n",
       "      <td>Activision Blizzard</td>\n",
       "      <td>reports</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Home Entertainment Software</td>\n",
       "      <td>Santa Monica, California</td>\n",
       "      <td>2015-08-31</td>\n",
       "      <td>718877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AYI</th>\n",
       "      <td>Acuity Brands Inc</td>\n",
       "      <td>reports</td>\n",
       "      <td>Industrials</td>\n",
       "      <td>Electrical Components &amp; Equipment</td>\n",
       "      <td>Atlanta, Georgia</td>\n",
       "      <td>2016-05-03</td>\n",
       "      <td>1144215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ADBE</th>\n",
       "      <td>Adobe Systems Inc</td>\n",
       "      <td>reports</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Application Software</td>\n",
       "      <td>San Jose, California</td>\n",
       "      <td>1997-05-05</td>\n",
       "      <td>796343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AMD</th>\n",
       "      <td>Advanced Micro Devices Inc</td>\n",
       "      <td>reports</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Semiconductors</td>\n",
       "      <td>Sunnyvale, California</td>\n",
       "      <td>2017-03-20</td>\n",
       "      <td>2488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AAP</th>\n",
       "      <td>Advance Auto Parts</td>\n",
       "      <td>reports</td>\n",
       "      <td>Consumer Discretionary</td>\n",
       "      <td>Automotive Retail</td>\n",
       "      <td>Roanoke, Virginia</td>\n",
       "      <td>2015-07-09</td>\n",
       "      <td>1158449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AES</th>\n",
       "      <td>AES Corp</td>\n",
       "      <td>reports</td>\n",
       "      <td>Utilities</td>\n",
       "      <td>Independent Power Producers &amp; Energy Traders</td>\n",
       "      <td>Arlington, Virginia</td>\n",
       "      <td>1998-10-02</td>\n",
       "      <td>874761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AET</th>\n",
       "      <td>Aetna Inc</td>\n",
       "      <td>reports</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Managed Health Care</td>\n",
       "      <td>Hartford, Connecticut</td>\n",
       "      <td>1976-06-30</td>\n",
       "      <td>1122304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AMG</th>\n",
       "      <td>Affiliated Managers Group Inc</td>\n",
       "      <td>reports</td>\n",
       "      <td>Financials</td>\n",
       "      <td>Asset Management &amp; Custody Banks</td>\n",
       "      <td>Beverly, Massachusetts</td>\n",
       "      <td>2014-07-01</td>\n",
       "      <td>1004434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AFL</th>\n",
       "      <td>AFLAC Inc</td>\n",
       "      <td>reports</td>\n",
       "      <td>Financials</td>\n",
       "      <td>Life &amp; Health Insurance</td>\n",
       "      <td>Columbus, Georgia</td>\n",
       "      <td>1999-05-28</td>\n",
       "      <td>4977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A</th>\n",
       "      <td>Agilent Technologies Inc</td>\n",
       "      <td>reports</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Health Care Equipment</td>\n",
       "      <td>Santa Clara, California</td>\n",
       "      <td>2000-06-05</td>\n",
       "      <td>1090872</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>APD</th>\n",
       "      <td>Air Products &amp; Chemicals Inc</td>\n",
       "      <td>reports</td>\n",
       "      <td>Materials</td>\n",
       "      <td>Industrial Gases</td>\n",
       "      <td>Allentown, Pennsylvania</td>\n",
       "      <td>1985-04-30</td>\n",
       "      <td>2969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AKAM</th>\n",
       "      <td>Akamai Technologies Inc</td>\n",
       "      <td>reports</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Internet Software &amp; Services</td>\n",
       "      <td>Cambridge, Massachusetts</td>\n",
       "      <td>2007-07-12</td>\n",
       "      <td>1086222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ALK</th>\n",
       "      <td>Alaska Air Group Inc</td>\n",
       "      <td>reports</td>\n",
       "      <td>Industrials</td>\n",
       "      <td>Airlines</td>\n",
       "      <td>Seattle, Washington</td>\n",
       "      <td>2016-05-13</td>\n",
       "      <td>766421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ALB</th>\n",
       "      <td>Albemarle Corp</td>\n",
       "      <td>reports</td>\n",
       "      <td>Materials</td>\n",
       "      <td>Specialty Chemicals</td>\n",
       "      <td>Baton Rouge, Louisiana</td>\n",
       "      <td>2016-07-01</td>\n",
       "      <td>915913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ARE</th>\n",
       "      <td>Alexandria Real Estate Equities Inc</td>\n",
       "      <td>reports</td>\n",
       "      <td>Real Estate</td>\n",
       "      <td>Office REITs</td>\n",
       "      <td>Pasadena, California</td>\n",
       "      <td>2017-03-20</td>\n",
       "      <td>1035443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ALXN</th>\n",
       "      <td>Alexion Pharmaceuticals</td>\n",
       "      <td>reports</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Biotechnology</td>\n",
       "      <td>Cheshire, Connecticut</td>\n",
       "      <td>2012-05-25</td>\n",
       "      <td>899866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ALGN</th>\n",
       "      <td>Align Technology</td>\n",
       "      <td>reports</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Health Care Supplies</td>\n",
       "      <td>San Jose, California</td>\n",
       "      <td>2017-06-19</td>\n",
       "      <td>1097149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ALLE</th>\n",
       "      <td>Allegion</td>\n",
       "      <td>reports</td>\n",
       "      <td>Industrials</td>\n",
       "      <td>Building Products</td>\n",
       "      <td>Dublin, Ireland</td>\n",
       "      <td>2013-12-02</td>\n",
       "      <td>1579241</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AGN</th>\n",
       "      <td>Allergan, Plc</td>\n",
       "      <td>reports</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Pharmaceuticals</td>\n",
       "      <td>Dublin, Ireland</td>\n",
       "      <td>1999-04-12</td>\n",
       "      <td>1578845</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ADS</th>\n",
       "      <td>Alliance Data Systems</td>\n",
       "      <td>reports</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Data Processing &amp; Outsourced Services</td>\n",
       "      <td>Plano, Texas</td>\n",
       "      <td>2013-12-23</td>\n",
       "      <td>1101215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LNT</th>\n",
       "      <td>Alliant Energy Corp</td>\n",
       "      <td>reports</td>\n",
       "      <td>Utilities</td>\n",
       "      <td>Electric Utilities</td>\n",
       "      <td>Madison, Wisconsin</td>\n",
       "      <td>2016-07-01</td>\n",
       "      <td>352541</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ALL</th>\n",
       "      <td>Allstate Corp</td>\n",
       "      <td>reports</td>\n",
       "      <td>Financials</td>\n",
       "      <td>Property &amp; Casualty Insurance</td>\n",
       "      <td>Northfield Township, Illinois</td>\n",
       "      <td>1995-07-13</td>\n",
       "      <td>899051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GOOGL</th>\n",
       "      <td>Alphabet Inc Class A</td>\n",
       "      <td>reports</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Internet Software &amp; Services</td>\n",
       "      <td>Mountain View, California</td>\n",
       "      <td>2014-04-03</td>\n",
       "      <td>1652044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GOOG</th>\n",
       "      <td>Alphabet Inc Class C</td>\n",
       "      <td>reports</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Internet Software &amp; Services</td>\n",
       "      <td>Mountain View, California</td>\n",
       "      <td>2006-04-03</td>\n",
       "      <td>1652044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MO</th>\n",
       "      <td>Altria Group Inc</td>\n",
       "      <td>reports</td>\n",
       "      <td>Consumer Staples</td>\n",
       "      <td>Tobacco</td>\n",
       "      <td>Richmond, Virginia</td>\n",
       "      <td>NaN</td>\n",
       "      <td>764180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AMZN</th>\n",
       "      <td>Amazon.com Inc.</td>\n",
       "      <td>reports</td>\n",
       "      <td>Consumer Discretionary</td>\n",
       "      <td>Internet &amp; Direct Marketing Retail</td>\n",
       "      <td>Seattle, Washington</td>\n",
       "      <td>2005-11-18</td>\n",
       "      <td>1018724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VIAB</th>\n",
       "      <td>Viacom Inc.</td>\n",
       "      <td>reports</td>\n",
       "      <td>Consumer Discretionary</td>\n",
       "      <td>Cable &amp; Satellite</td>\n",
       "      <td>New York, New York</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1339947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>V</th>\n",
       "      <td>Visa Inc.</td>\n",
       "      <td>reports</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Internet Software &amp; Services</td>\n",
       "      <td>San Francisco, California</td>\n",
       "      <td>2009-12-21</td>\n",
       "      <td>1403161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VNO</th>\n",
       "      <td>Vornado Realty Trust</td>\n",
       "      <td>reports</td>\n",
       "      <td>Real Estate</td>\n",
       "      <td>Office REITs</td>\n",
       "      <td>New York, New York</td>\n",
       "      <td>NaN</td>\n",
       "      <td>899689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VMC</th>\n",
       "      <td>Vulcan Materials</td>\n",
       "      <td>reports</td>\n",
       "      <td>Materials</td>\n",
       "      <td>Construction Materials</td>\n",
       "      <td>Birmingham, Alabama</td>\n",
       "      <td>1999-06-30</td>\n",
       "      <td>1396009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WMT</th>\n",
       "      <td>Wal-Mart Stores</td>\n",
       "      <td>reports</td>\n",
       "      <td>Consumer Staples</td>\n",
       "      <td>Hypermarkets &amp; Super Centers</td>\n",
       "      <td>Bentonville, Arkansas</td>\n",
       "      <td>1982-08-31</td>\n",
       "      <td>104169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WBA</th>\n",
       "      <td>Walgreens Boots Alliance</td>\n",
       "      <td>reports</td>\n",
       "      <td>Consumer Staples</td>\n",
       "      <td>Drug Retail</td>\n",
       "      <td>Deerfield, Illinois</td>\n",
       "      <td>1979-12-31</td>\n",
       "      <td>1618921</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIS</th>\n",
       "      <td>The Walt Disney Company</td>\n",
       "      <td>reports</td>\n",
       "      <td>Consumer Discretionary</td>\n",
       "      <td>Cable &amp; Satellite</td>\n",
       "      <td>Burbank, California</td>\n",
       "      <td>1976-06-30</td>\n",
       "      <td>1001039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WM</th>\n",
       "      <td>Waste Management Inc.</td>\n",
       "      <td>reports</td>\n",
       "      <td>Industrials</td>\n",
       "      <td>Environmental &amp; Facilities Services</td>\n",
       "      <td>Houston, Texas</td>\n",
       "      <td>NaN</td>\n",
       "      <td>823768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WAT</th>\n",
       "      <td>Waters Corporation</td>\n",
       "      <td>reports</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Health Care Distributors</td>\n",
       "      <td>Milford, Massachusetts</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1000697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WEC</th>\n",
       "      <td>Wec Energy Group Inc</td>\n",
       "      <td>reports</td>\n",
       "      <td>Utilities</td>\n",
       "      <td>Electric Utilities</td>\n",
       "      <td>Milwaukee, Wisconsin</td>\n",
       "      <td>2008-10-31</td>\n",
       "      <td>783325</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WFC</th>\n",
       "      <td>Wells Fargo</td>\n",
       "      <td>reports</td>\n",
       "      <td>Financials</td>\n",
       "      <td>Diversified Banks</td>\n",
       "      <td>San Francisco, California</td>\n",
       "      <td>1976-06-30</td>\n",
       "      <td>72971</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WELL</th>\n",
       "      <td>Welltower Inc.</td>\n",
       "      <td>reports</td>\n",
       "      <td>Real Estate</td>\n",
       "      <td>Health Care REITs</td>\n",
       "      <td>Toledo, Ohio</td>\n",
       "      <td>2009-01-30</td>\n",
       "      <td>766704</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WDC</th>\n",
       "      <td>Western Digital</td>\n",
       "      <td>reports</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Technology Hardware, Storage &amp; Peripherals</td>\n",
       "      <td>Irvine, California</td>\n",
       "      <td>2009-07-01</td>\n",
       "      <td>106040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WU</th>\n",
       "      <td>Western Union Co</td>\n",
       "      <td>reports</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Internet Software &amp; Services</td>\n",
       "      <td>Englewood, Colorado</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1365135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WRK</th>\n",
       "      <td>WestRock Company</td>\n",
       "      <td>reports</td>\n",
       "      <td>Materials</td>\n",
       "      <td>Paper Packaging</td>\n",
       "      <td>Richmond, Virginia</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1636023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WY</th>\n",
       "      <td>Weyerhaeuser Corp.</td>\n",
       "      <td>reports</td>\n",
       "      <td>Real Estate</td>\n",
       "      <td>Specialized REITs</td>\n",
       "      <td>Federal Way, Washington</td>\n",
       "      <td>NaN</td>\n",
       "      <td>106535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WHR</th>\n",
       "      <td>Whirlpool Corp.</td>\n",
       "      <td>reports</td>\n",
       "      <td>Consumer Discretionary</td>\n",
       "      <td>Household Appliances</td>\n",
       "      <td>Benton Harbor, Michigan</td>\n",
       "      <td>NaN</td>\n",
       "      <td>106640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WMB</th>\n",
       "      <td>Williams Cos.</td>\n",
       "      <td>reports</td>\n",
       "      <td>Energy</td>\n",
       "      <td>Oil &amp; Gas Storage &amp; Transportation</td>\n",
       "      <td>Tulsa, Oklahoma</td>\n",
       "      <td>1975-03-31</td>\n",
       "      <td>107263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WLTW</th>\n",
       "      <td>Willis Towers Watson</td>\n",
       "      <td>reports</td>\n",
       "      <td>Financials</td>\n",
       "      <td>Insurance Brokers</td>\n",
       "      <td>London, United Kingdom</td>\n",
       "      <td>2016-01-05</td>\n",
       "      <td>1140536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WYN</th>\n",
       "      <td>Wyndham Worldwide</td>\n",
       "      <td>reports</td>\n",
       "      <td>Consumer Discretionary</td>\n",
       "      <td>Hotels, Resorts &amp; Cruise Lines</td>\n",
       "      <td>Parsippany, New Jersey</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1361658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WYNN</th>\n",
       "      <td>Wynn Resorts Ltd</td>\n",
       "      <td>reports</td>\n",
       "      <td>Consumer Discretionary</td>\n",
       "      <td>Casinos &amp; Gaming</td>\n",
       "      <td>Paradise, Nevada</td>\n",
       "      <td>2008-11-14</td>\n",
       "      <td>1174922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XEL</th>\n",
       "      <td>Xcel Energy Inc</td>\n",
       "      <td>reports</td>\n",
       "      <td>Utilities</td>\n",
       "      <td>Multi-Utilities</td>\n",
       "      <td>Minneapolis, Minnesota</td>\n",
       "      <td>NaN</td>\n",
       "      <td>72903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XRX</th>\n",
       "      <td>Xerox Corp.</td>\n",
       "      <td>reports</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Technology Hardware, Storage &amp; Peripherals</td>\n",
       "      <td>Norwalk, Connecticut</td>\n",
       "      <td>NaN</td>\n",
       "      <td>108772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XLNX</th>\n",
       "      <td>Xilinx Inc</td>\n",
       "      <td>reports</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Semiconductors</td>\n",
       "      <td>San Jose, California</td>\n",
       "      <td>1999-11-08</td>\n",
       "      <td>743988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XL</th>\n",
       "      <td>XL Capital</td>\n",
       "      <td>reports</td>\n",
       "      <td>Financials</td>\n",
       "      <td>Property &amp; Casualty Insurance</td>\n",
       "      <td>Hamilton, Bermuda</td>\n",
       "      <td>NaN</td>\n",
       "      <td>875159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XYL</th>\n",
       "      <td>Xylem Inc.</td>\n",
       "      <td>reports</td>\n",
       "      <td>Industrials</td>\n",
       "      <td>Industrial Machinery</td>\n",
       "      <td>White Plains, New York</td>\n",
       "      <td>2011-11-01</td>\n",
       "      <td>1524472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>YUM</th>\n",
       "      <td>Yum! Brands Inc</td>\n",
       "      <td>reports</td>\n",
       "      <td>Consumer Discretionary</td>\n",
       "      <td>Restaurants</td>\n",
       "      <td>Louisville, Kentucky</td>\n",
       "      <td>1997-10-06</td>\n",
       "      <td>1041061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ZBH</th>\n",
       "      <td>Zimmer Biomet Holdings</td>\n",
       "      <td>reports</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Health Care Equipment</td>\n",
       "      <td>Warsaw, Indiana</td>\n",
       "      <td>2001-08-07</td>\n",
       "      <td>1136869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ZION</th>\n",
       "      <td>Zions Bancorp</td>\n",
       "      <td>reports</td>\n",
       "      <td>Financials</td>\n",
       "      <td>Regional Banks</td>\n",
       "      <td>Salt Lake City, Utah</td>\n",
       "      <td>2001-06-22</td>\n",
       "      <td>109380</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ZTS</th>\n",
       "      <td>Zoetis</td>\n",
       "      <td>reports</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Pharmaceuticals</td>\n",
       "      <td>Florham Park, New Jersey</td>\n",
       "      <td>2013-06-21</td>\n",
       "      <td>1555280</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>505 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          Security SEC filings  \\\n",
       "Ticker symbol                                                    \n",
       "MMM                                     3M Company     reports   \n",
       "ABT                            Abbott Laboratories     reports   \n",
       "ABBV                                   AbbVie Inc.     reports   \n",
       "ACN                                  Accenture plc     reports   \n",
       "ATVI                           Activision Blizzard     reports   \n",
       "AYI                              Acuity Brands Inc     reports   \n",
       "ADBE                             Adobe Systems Inc     reports   \n",
       "AMD                     Advanced Micro Devices Inc     reports   \n",
       "AAP                             Advance Auto Parts     reports   \n",
       "AES                                       AES Corp     reports   \n",
       "AET                                      Aetna Inc     reports   \n",
       "AMG                  Affiliated Managers Group Inc     reports   \n",
       "AFL                                      AFLAC Inc     reports   \n",
       "A                         Agilent Technologies Inc     reports   \n",
       "APD                   Air Products & Chemicals Inc     reports   \n",
       "AKAM                       Akamai Technologies Inc     reports   \n",
       "ALK                           Alaska Air Group Inc     reports   \n",
       "ALB                                 Albemarle Corp     reports   \n",
       "ARE            Alexandria Real Estate Equities Inc     reports   \n",
       "ALXN                       Alexion Pharmaceuticals     reports   \n",
       "ALGN                              Align Technology     reports   \n",
       "ALLE                                      Allegion     reports   \n",
       "AGN                                  Allergan, Plc     reports   \n",
       "ADS                          Alliance Data Systems     reports   \n",
       "LNT                            Alliant Energy Corp     reports   \n",
       "ALL                                  Allstate Corp     reports   \n",
       "GOOGL                         Alphabet Inc Class A     reports   \n",
       "GOOG                          Alphabet Inc Class C     reports   \n",
       "MO                                Altria Group Inc     reports   \n",
       "AMZN                               Amazon.com Inc.     reports   \n",
       "...                                            ...         ...   \n",
       "VIAB                                   Viacom Inc.     reports   \n",
       "V                                        Visa Inc.     reports   \n",
       "VNO                           Vornado Realty Trust     reports   \n",
       "VMC                               Vulcan Materials     reports   \n",
       "WMT                                Wal-Mart Stores     reports   \n",
       "WBA                       Walgreens Boots Alliance     reports   \n",
       "DIS                        The Walt Disney Company     reports   \n",
       "WM                           Waste Management Inc.     reports   \n",
       "WAT                             Waters Corporation     reports   \n",
       "WEC                           Wec Energy Group Inc     reports   \n",
       "WFC                                    Wells Fargo     reports   \n",
       "WELL                                Welltower Inc.     reports   \n",
       "WDC                                Western Digital     reports   \n",
       "WU                                Western Union Co     reports   \n",
       "WRK                               WestRock Company     reports   \n",
       "WY                              Weyerhaeuser Corp.     reports   \n",
       "WHR                                Whirlpool Corp.     reports   \n",
       "WMB                                  Williams Cos.     reports   \n",
       "WLTW                          Willis Towers Watson     reports   \n",
       "WYN                              Wyndham Worldwide     reports   \n",
       "WYNN                              Wynn Resorts Ltd     reports   \n",
       "XEL                                Xcel Energy Inc     reports   \n",
       "XRX                                    Xerox Corp.     reports   \n",
       "XLNX                                    Xilinx Inc     reports   \n",
       "XL                                      XL Capital     reports   \n",
       "XYL                                     Xylem Inc.     reports   \n",
       "YUM                                Yum! Brands Inc     reports   \n",
       "ZBH                         Zimmer Biomet Holdings     reports   \n",
       "ZION                                 Zions Bancorp     reports   \n",
       "ZTS                                         Zoetis     reports   \n",
       "\n",
       "                          GICS Sector  \\\n",
       "Ticker symbol                           \n",
       "MMM                       Industrials   \n",
       "ABT                       Health Care   \n",
       "ABBV                      Health Care   \n",
       "ACN            Information Technology   \n",
       "ATVI           Information Technology   \n",
       "AYI                       Industrials   \n",
       "ADBE           Information Technology   \n",
       "AMD            Information Technology   \n",
       "AAP            Consumer Discretionary   \n",
       "AES                         Utilities   \n",
       "AET                       Health Care   \n",
       "AMG                        Financials   \n",
       "AFL                        Financials   \n",
       "A                         Health Care   \n",
       "APD                         Materials   \n",
       "AKAM           Information Technology   \n",
       "ALK                       Industrials   \n",
       "ALB                         Materials   \n",
       "ARE                       Real Estate   \n",
       "ALXN                      Health Care   \n",
       "ALGN                      Health Care   \n",
       "ALLE                      Industrials   \n",
       "AGN                       Health Care   \n",
       "ADS            Information Technology   \n",
       "LNT                         Utilities   \n",
       "ALL                        Financials   \n",
       "GOOGL          Information Technology   \n",
       "GOOG           Information Technology   \n",
       "MO                   Consumer Staples   \n",
       "AMZN           Consumer Discretionary   \n",
       "...                               ...   \n",
       "VIAB           Consumer Discretionary   \n",
       "V              Information Technology   \n",
       "VNO                       Real Estate   \n",
       "VMC                         Materials   \n",
       "WMT                  Consumer Staples   \n",
       "WBA                  Consumer Staples   \n",
       "DIS            Consumer Discretionary   \n",
       "WM                        Industrials   \n",
       "WAT                       Health Care   \n",
       "WEC                         Utilities   \n",
       "WFC                        Financials   \n",
       "WELL                      Real Estate   \n",
       "WDC            Information Technology   \n",
       "WU             Information Technology   \n",
       "WRK                         Materials   \n",
       "WY                        Real Estate   \n",
       "WHR            Consumer Discretionary   \n",
       "WMB                            Energy   \n",
       "WLTW                       Financials   \n",
       "WYN            Consumer Discretionary   \n",
       "WYNN           Consumer Discretionary   \n",
       "XEL                         Utilities   \n",
       "XRX            Information Technology   \n",
       "XLNX           Information Technology   \n",
       "XL                         Financials   \n",
       "XYL                       Industrials   \n",
       "YUM            Consumer Discretionary   \n",
       "ZBH                       Health Care   \n",
       "ZION                       Financials   \n",
       "ZTS                       Health Care   \n",
       "\n",
       "                                          GICS Sub Industry  \\\n",
       "Ticker symbol                                                 \n",
       "MMM                                Industrial Conglomerates   \n",
       "ABT                                   Health Care Equipment   \n",
       "ABBV                                        Pharmaceuticals   \n",
       "ACN                          IT Consulting & Other Services   \n",
       "ATVI                            Home Entertainment Software   \n",
       "AYI                       Electrical Components & Equipment   \n",
       "ADBE                                   Application Software   \n",
       "AMD                                          Semiconductors   \n",
       "AAP                                       Automotive Retail   \n",
       "AES            Independent Power Producers & Energy Traders   \n",
       "AET                                     Managed Health Care   \n",
       "AMG                        Asset Management & Custody Banks   \n",
       "AFL                                 Life & Health Insurance   \n",
       "A                                     Health Care Equipment   \n",
       "APD                                        Industrial Gases   \n",
       "AKAM                           Internet Software & Services   \n",
       "ALK                                                Airlines   \n",
       "ALB                                     Specialty Chemicals   \n",
       "ARE                                            Office REITs   \n",
       "ALXN                                          Biotechnology   \n",
       "ALGN                                   Health Care Supplies   \n",
       "ALLE                                      Building Products   \n",
       "AGN                                         Pharmaceuticals   \n",
       "ADS                   Data Processing & Outsourced Services   \n",
       "LNT                                      Electric Utilities   \n",
       "ALL                           Property & Casualty Insurance   \n",
       "GOOGL                          Internet Software & Services   \n",
       "GOOG                           Internet Software & Services   \n",
       "MO                                                  Tobacco   \n",
       "AMZN                     Internet & Direct Marketing Retail   \n",
       "...                                                     ...   \n",
       "VIAB                                      Cable & Satellite   \n",
       "V                              Internet Software & Services   \n",
       "VNO                                            Office REITs   \n",
       "VMC                                  Construction Materials   \n",
       "WMT                            Hypermarkets & Super Centers   \n",
       "WBA                                             Drug Retail   \n",
       "DIS                                       Cable & Satellite   \n",
       "WM                      Environmental & Facilities Services   \n",
       "WAT                                Health Care Distributors   \n",
       "WEC                                      Electric Utilities   \n",
       "WFC                                       Diversified Banks   \n",
       "WELL                                      Health Care REITs   \n",
       "WDC              Technology Hardware, Storage & Peripherals   \n",
       "WU                             Internet Software & Services   \n",
       "WRK                                         Paper Packaging   \n",
       "WY                                        Specialized REITs   \n",
       "WHR                                    Household Appliances   \n",
       "WMB                      Oil & Gas Storage & Transportation   \n",
       "WLTW                                      Insurance Brokers   \n",
       "WYN                          Hotels, Resorts & Cruise Lines   \n",
       "WYNN                                       Casinos & Gaming   \n",
       "XEL                                         Multi-Utilities   \n",
       "XRX              Technology Hardware, Storage & Peripherals   \n",
       "XLNX                                         Semiconductors   \n",
       "XL                            Property & Casualty Insurance   \n",
       "XYL                                    Industrial Machinery   \n",
       "YUM                                             Restaurants   \n",
       "ZBH                                   Health Care Equipment   \n",
       "ZION                                         Regional Banks   \n",
       "ZTS                                         Pharmaceuticals   \n",
       "\n",
       "                     Address of Headquarters Date first added[3][4]      CIK  \n",
       "Ticker symbol                                                                 \n",
       "MMM                      St. Paul, Minnesota                    NaN    66740  \n",
       "ABT                  North Chicago, Illinois             1964-03-31     1800  \n",
       "ABBV                 North Chicago, Illinois             2012-12-31  1551152  \n",
       "ACN                          Dublin, Ireland             2011-07-06  1467373  \n",
       "ATVI                Santa Monica, California             2015-08-31   718877  \n",
       "AYI                         Atlanta, Georgia             2016-05-03  1144215  \n",
       "ADBE                    San Jose, California             1997-05-05   796343  \n",
       "AMD                    Sunnyvale, California             2017-03-20     2488  \n",
       "AAP                        Roanoke, Virginia             2015-07-09  1158449  \n",
       "AES                      Arlington, Virginia             1998-10-02   874761  \n",
       "AET                    Hartford, Connecticut             1976-06-30  1122304  \n",
       "AMG                   Beverly, Massachusetts             2014-07-01  1004434  \n",
       "AFL                        Columbus, Georgia             1999-05-28     4977  \n",
       "A                    Santa Clara, California             2000-06-05  1090872  \n",
       "APD                  Allentown, Pennsylvania             1985-04-30     2969  \n",
       "AKAM                Cambridge, Massachusetts             2007-07-12  1086222  \n",
       "ALK                      Seattle, Washington             2016-05-13   766421  \n",
       "ALB                   Baton Rouge, Louisiana             2016-07-01   915913  \n",
       "ARE                     Pasadena, California             2017-03-20  1035443  \n",
       "ALXN                   Cheshire, Connecticut             2012-05-25   899866  \n",
       "ALGN                    San Jose, California             2017-06-19  1097149  \n",
       "ALLE                         Dublin, Ireland             2013-12-02  1579241  \n",
       "AGN                          Dublin, Ireland             1999-04-12  1578845  \n",
       "ADS                             Plano, Texas             2013-12-23  1101215  \n",
       "LNT                       Madison, Wisconsin             2016-07-01   352541  \n",
       "ALL            Northfield Township, Illinois             1995-07-13   899051  \n",
       "GOOGL              Mountain View, California             2014-04-03  1652044  \n",
       "GOOG               Mountain View, California             2006-04-03  1652044  \n",
       "MO                        Richmond, Virginia                    NaN   764180  \n",
       "AMZN                     Seattle, Washington             2005-11-18  1018724  \n",
       "...                                      ...                    ...      ...  \n",
       "VIAB                      New York, New York                    NaN  1339947  \n",
       "V                  San Francisco, California             2009-12-21  1403161  \n",
       "VNO                       New York, New York                    NaN   899689  \n",
       "VMC                      Birmingham, Alabama             1999-06-30  1396009  \n",
       "WMT                    Bentonville, Arkansas             1982-08-31   104169  \n",
       "WBA                      Deerfield, Illinois             1979-12-31  1618921  \n",
       "DIS                      Burbank, California             1976-06-30  1001039  \n",
       "WM                            Houston, Texas                    NaN   823768  \n",
       "WAT                   Milford, Massachusetts                    NaN  1000697  \n",
       "WEC                     Milwaukee, Wisconsin             2008-10-31   783325  \n",
       "WFC                San Francisco, California             1976-06-30    72971  \n",
       "WELL                            Toledo, Ohio             2009-01-30   766704  \n",
       "WDC                       Irvine, California             2009-07-01   106040  \n",
       "WU                       Englewood, Colorado                    NaN  1365135  \n",
       "WRK                       Richmond, Virginia                    NaN  1636023  \n",
       "WY                   Federal Way, Washington                    NaN   106535  \n",
       "WHR                  Benton Harbor, Michigan                    NaN   106640  \n",
       "WMB                          Tulsa, Oklahoma             1975-03-31   107263  \n",
       "WLTW                  London, United Kingdom             2016-01-05  1140536  \n",
       "WYN                   Parsippany, New Jersey                    NaN  1361658  \n",
       "WYNN                        Paradise, Nevada             2008-11-14  1174922  \n",
       "XEL                   Minneapolis, Minnesota                    NaN    72903  \n",
       "XRX                     Norwalk, Connecticut                    NaN   108772  \n",
       "XLNX                    San Jose, California             1999-11-08   743988  \n",
       "XL                         Hamilton, Bermuda                    NaN   875159  \n",
       "XYL                   White Plains, New York             2011-11-01  1524472  \n",
       "YUM                     Louisville, Kentucky             1997-10-06  1041061  \n",
       "ZBH                          Warsaw, Indiana             2001-08-07  1136869  \n",
       "ZION                    Salt Lake City, Utah             2001-06-22   109380  \n",
       "ZTS                 Florham Park, New Jersey             2013-06-21  1555280  \n",
       "\n",
       "[505 rows x 7 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get Current S&P 500 List, Stock Ticker, and CIK\n",
    "wiki_url = \"https://en.wikipedia.org/wiki/List_of_S%26P_500_companies\"\n",
    "cik_df = pd.read_html(wiki_url,header=[0],index_col=0)[0]\n",
    "cik_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'Pickles/sec_links_df.pkl'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-8357558ff6e9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mdf_list\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mget_sec_docs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcik\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mticker\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mcrawled_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mcrawled_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_pickle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Pickles/sec_links_df.pkl\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36mto_pickle\u001b[0;34m(self, path, compression, protocol)\u001b[0m\n\u001b[1;32m   1562\u001b[0m         \u001b[0;32mfrom\u001b[0m \u001b[0mpandas\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpickle\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mto_pickle\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1563\u001b[0m         return to_pickle(self, path, compression=compression,\n\u001b[0;32m-> 1564\u001b[0;31m                          protocol=protocol)\n\u001b[0m\u001b[1;32m   1565\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1566\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mto_clipboard\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexcel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msep\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/pandas/io/pickle.py\u001b[0m in \u001b[0;36mto_pickle\u001b[0;34m(obj, path, compression, protocol)\u001b[0m\n\u001b[1;32m     39\u001b[0m     f, fh = _get_handle(path, 'wb',\n\u001b[1;32m     40\u001b[0m                         \u001b[0mcompression\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minferred_compression\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 41\u001b[0;31m                         is_text=False)\n\u001b[0m\u001b[1;32m     42\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mprotocol\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m         \u001b[0mprotocol\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpkl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mHIGHEST_PROTOCOL\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/pandas/io/common.py\u001b[0m in \u001b[0;36m_get_handle\u001b[0;34m(path_or_buf, mode, encoding, compression, memory_map, is_text)\u001b[0m\n\u001b[1;32m    394\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    395\u001b[0m             \u001b[0;31m# Python 3 and binary mode\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 396\u001b[0;31m             \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath_or_buf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    397\u001b[0m         \u001b[0mhandles\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    398\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'Pickles/sec_links_df.pkl'"
     ]
    }
   ],
   "source": [
    "df_list = []\n",
    "company_list = cik_df['CIK'].to_dict()\n",
    "for (ticker,cik) in company_list.items():\n",
    "    df_list.append(get_sec_docs(cik,ticker))\n",
    "crawled_df = pd.concat(df_list,axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "crawled_df.to_pickle(\"Pickles/doc_links_df.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "crawled_df['text'], crawled_df['release_date'] = zip(*crawled_df['txt_link'].apply(extract_text))\n",
    "crawled_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "crawled_df.to_pickle(\"Pickles/doc_texts.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'crawled_df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-e3babeb2a217>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mcrawled_df\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'crawled_df' is not defined"
     ]
    }
   ],
   "source": [
    "crawled_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
